package com.itheima.newsQQ;

import com.google.gson.Gson;
import com.itheima.dao.NewsDao;
import com.itheima.pojo.News;
import com.itheima.pojo.SpiderFinal;
import com.itheima.tools.HttpClientUtils;
import com.itheima.tools.IdWorker;
import com.itheima.tools.JedisUtil;
import redis.clients.jedis.Jedis;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * 腾讯新闻爬取
 */
public class NewsQQ_Spider {

    /**
     * Gson
     */
    private static Gson gson = new Gson();

    /**
     * 分布式id生成器
     */
    private static IdWorker idWorker = new IdWorker(0, 0);
    /**
     * 创建所需对象
     */
    private static NewsDao newsDao = new NewsDao();



    public static void main(String[] args) throws IOException {
        //1. 确定爬取的url
        String hotUrl = "https://pacaio.match.qq.com/irs/rcd?cid=137&token=d0f13d594edfc180f5bf6b845456f3ea&id=&ext=ent&num=60";

        String noHotUrl = "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page=0";


        //2. 发送请求，获取json数据
        /*String hotJson = HttpClientUtils.doGet(hotUrl);
        String nohotJson = HttpClientUtils.doGet(noHotUrl);
        System.out.println("热点数据：" + hotJson);
        System.out.println("非热点数据：" + nohotJson);*/


        //3. 解析json数据，保存数据库
        /*List<News> hotNewsList = parseJsonNews(hotJson);
        List<News> nohotNewsList = parseJsonNews(nohotJson);*/

        //4. 保存到数据库
        /*saveNewsToDb(hotNewsList);
        saveNewsToDb(nohotNewsList);*/

        pageTencent(hotUrl, noHotUrl);
    }


    /**
     * 腾讯娱乐新闻的分页爬取方法
     */
    private static void pageTencent(String hotUrl, String noHotUrl) throws IOException {

        //1. 先处理热点新闻数据(就一页数据)
        String hotJson = HttpClientUtils.doGet(hotUrl);
        List<News> hotNewsList = parseJsonNews(hotJson);
        saveNewsToDb(hotNewsList);


        //2.非热点新闻数据需要处理分页
        int page = 1;
        while (true) {
            String nohotJson = HttpClientUtils.doGet(noHotUrl);

            List<News> nohotNewsList = parseJsonNews(nohotJson);

            //跳出循环的逻辑：没有json数据的时候
            if (nohotNewsList == null || nohotNewsList.size() == 0) {
                break;
            }

            saveNewsToDb(nohotNewsList);


            //构造下一页的url地址
            noHotUrl = "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page=" + page;
            page++;

        }

    }


    /**
     * 保存新闻列表数据到数据库中
     *
     * @param newsList ： 新闻列表
     */
    private static void saveNewsToDb(List<News> newsList) {

        for (News news : newsList) {
            newsDao.saveNews(news);

            //将爬取过的url保存到redis的set集合中
            Jedis jedis = JedisUtil.getJedis();

            jedis.sadd(SpiderFinal.SPIEDR_NEWSQQ, news.getDocurl());
            jedis.close();
        }
    }

    /**
     * 解析json数据，返回新闻列表
     *
     * @param jsonNews
     */
    private static List<News> parseJsonNews(String jsonNews) {

        List<News> newsList = new ArrayList<News>();

        //1. jsonNews ------->  Map<String,object>
        Map<String, Object> map = gson.fromJson(jsonNews, Map.class);

        //2.获取新闻列表数据
        List<Map<String, Object>> data = (List<Map<String, Object>>) map.get("data");
        for (Map<String, Object> news : data) {

            News newsBean = new News();
            //获取url
            String docUrl = (String) news.get("vurl");

            //进行过滤，如果是视频文件，排除掉
            if (docUrl.contains("video")) {
                continue;
            }
            //判断这个url是否已经爬取过
            boolean hasParsed = isParsedDocUrl(docUrl);
            if (hasParsed) {
                continue;
            }

            System.out.println("url：" + docUrl);

            //标题
            String title = (String) news.get("title");
            System.out.println("标题：" + title);

            //时间
            String time = (String) news.get("update_time");


            //来源
            String source = (String) news.get("source");

            //编辑==来源

            //内容
            String content = (String) news.get("intro");

            newsBean.setId(idWorker.nextId() + "");
            newsBean.setTitle(title);
            newsBean.setSource(source);
            newsBean.setTime(time);
            newsBean.setContent(content);
            newsBean.setEditor(source);
            newsBean.setDocurl(docUrl);

            //添加到list列表
            newsList.add(newsBean);
        }

        return newsList;
    }

    /**
     * 判断给定的url是否已经爬取过
     *
     * @param docUrl ： 判断的url
     * @return
     */
    private static boolean isParsedDocUrl(String docUrl) {
        //1. 获取jedis连接
        Jedis jedis = JedisUtil.getJedis();

        //2. 判断是否在set集合中
        Boolean sismember = jedis.sismember(SpiderFinal.SPIEDR_NEWSQQ, docUrl);

        //3.关闭
        jedis.close();

        //4.返回
        return sismember;
    }


}



